import asyncio
import re
from bs4 import BeautifulSoup
from lxml import etree
from aiohttp import ClientSession
from pymongo import MongoClient

url = 'https://nanjing.qfang.com/sale/f%s'


# 保存数据
def save_text(task_obj):
    # 拆分数据
    r_text = etree.HTML(task_obj.result())
    title = r_text.xpath('//div[@class="list-result"]//a[@class="house-title fl"]/text()')
    house_type = r_text.xpath('//div[@class="list-result"]//div[@class="house-metas clearfix"]/p[1]/text()')
    area = r_text.xpath('//div[@class="list-result"]//div[@class="house-metas clearfix"]/p[2]/text()')
    plies = r_text.xpath('//div[@class="list-result"]//div[@class="house-metas clearfix"]/p[4]/text()')
    face = r_text.xpath('//div[@class="list-result"]//div[@class="house-metas clearfix"]/p[last()-1]/text()')
    time = r_text.xpath('//div[@class="list-result"]//div[@class="house-metas clearfix"]/p[last()]/text()')
    other = BeautifulSoup(task_obj.result(), features="lxml").find_all(**{'class': 'house-tags clearfix'})
    big = r_text.xpath('//div[@class="list-result"]//div[@class="list-price"]//span[1]/text()')
    small = r_text.xpath('//div[@class="list-result"]//div[@class="list-price"]/p[@class="smaller"]/text()')
    # 打印数据数量
    print('title', len(title), 'house_type', len(house_type), 'area', len(area), 'plies', len(plies), 'face', len(face),
          'time', len(time), 'other', len(other), 'big', len(big), 'small', len(small))
    # 数据重组
    text_list = []
    for i in range(len(title)):
        other_list = re.findall(r'<span>(.+)</span>', str(other[i]))+re.findall(r'<p class="default fl">(.+)</p>', str(other[i]))
        text_list.append(
            {'title': title[i],
             'house_type': house_type[i],
             'area': area[i],
             'plies': plies[i],
             'face': face[i],
             'time': time[i],
             'other': other_list,
             'big': big[i] + '万',
             'small': small[i]})
    # 数据存贮
    client = MongoClient(host='127.0.0.1', port=27017)
    collent = client['qfang']['2020_11_12']
    collent.insert_many(text_list)


# 发送请求
async def RTX(num):
    async with ClientSession() as session:
        async with session.get(url % num) as response:
            return await response.text()


# 初始化
async def main():
    # 创建多个Task，并添加回调函数
    tasks_list = []
    for i in range(1, 11):
        task_qfang = asyncio.create_task(RTX(i))
        task_qfang.add_done_callback(save_text)
        tasks_list.append(task_qfang)
        await asyncio.wait(tasks_list)


if __name__ == '__main__':
    asyncio.run(main())
